{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "save_path = \"/data/zyang/datasets/ml-1m/\"\n",
    "# trai.to_pickle(save_path+\"train_ood2.pkl\")\n",
    "train_ = pd.read_pickle(save_path+\"train_ood2.pkl\")\n",
    "valid_ = pd.read_pickle(save_path+\"valid_ood2.pkl\")\n",
    "test_ = pd.read_pickle(save_path+\"test_ood2.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_info = train_.groupby('uid').agg({\"label\":'count'})\n",
    "user_info.shape\n",
    "train_user = np.array(list(user_info.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label    5.0\n",
       "Name: 0.2, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info.quantile(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(635, 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info = user_info[user_info.label>3] \n",
    "user_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_info = train_.groupby('iid').agg({\"label\":'count'})\n",
    "item_info.shape\n",
    "\n",
    "train_item = np.array(list(item_info.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3087.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>10.978620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>12.641559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>154.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             label\n",
       "count  3087.000000\n",
       "mean     10.978620\n",
       "std      12.641559\n",
       "min       1.000000\n",
       "25%       2.000000\n",
       "50%       6.000000\n",
       "75%      15.000000\n",
       "max     154.000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_info.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(label    3.0\n",
       " Name: 0.3, dtype: float64,\n",
       " (3087, 1))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_info.quantile(0.3),item_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2058, 1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_info = item_info[item_info.label>3] \n",
    "item_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "warm_user = np.array(list(user_info.index))\n",
    "warm_item = np.array(list(item_info.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((635,), (2058,), (740,), (3087,))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "warm_user.shape, warm_item.shape, train_user.shape, train_item.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_['warm'] = test_[['uid','iid']].apply(lambda x: x.uid in warm_user and x.iid in warm_item, axis=1).astype(\"int\")\n",
    "test_['cold'] = test_[['uid','iid']].apply(lambda x: x.uid not in train_user and x.iid not in train_item, axis=1).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7331, 11)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>iid</th>\n",
       "      <th>label</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>his</th>\n",
       "      <th>his_title</th>\n",
       "      <th>title</th>\n",
       "      <th>flag</th>\n",
       "      <th>not_cold</th>\n",
       "      <th>warm</th>\n",
       "      <th>cold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>4</td>\n",
       "      <td>96</td>\n",
       "      <td>1</td>\n",
       "      <td>1040544350</td>\n",
       "      <td>[0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...</td>\n",
       "      <td>[, Star Wars: Episode IV - A New Hope (1977), ...</td>\n",
       "      <td>Deconstructing Harry (1997)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>4</td>\n",
       "      <td>97</td>\n",
       "      <td>1</td>\n",
       "      <td>1040544350</td>\n",
       "      <td>[0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...</td>\n",
       "      <td>[, Star Wars: Episode IV - A New Hope (1977), ...</td>\n",
       "      <td>Indecent Proposal (1993)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>4</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "      <td>1040544494</td>\n",
       "      <td>[0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...</td>\n",
       "      <td>[, Star Wars: Episode IV - A New Hope (1977), ...</td>\n",
       "      <td>Big Daddy (1999)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>4</td>\n",
       "      <td>99</td>\n",
       "      <td>1</td>\n",
       "      <td>1040544521</td>\n",
       "      <td>[0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...</td>\n",
       "      <td>[, Star Wars: Episode IV - A New Hope (1977), ...</td>\n",
       "      <td>But I'm a Cheerleader (1999)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>4</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>1040544564</td>\n",
       "      <td>[0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...</td>\n",
       "      <td>[, Star Wars: Episode IV - A New Hope (1977), ...</td>\n",
       "      <td>Desperately Seeking Susan (1985)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     uid  iid  label   timestamp  \\\n",
       "97     4   96      1  1040544350   \n",
       "98     4   97      1  1040544350   \n",
       "99     4   98      0  1040544494   \n",
       "100    4   99      1  1040544521   \n",
       "101    4  100      1  1040544564   \n",
       "\n",
       "                                                   his  \\\n",
       "97   [0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...   \n",
       "98   [0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...   \n",
       "99   [0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...   \n",
       "100  [0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...   \n",
       "101  [0, 38, 39, 40, 42, 43, 44, 45, 46, 47, 48, 50...   \n",
       "\n",
       "                                             his_title  \\\n",
       "97   [, Star Wars: Episode IV - A New Hope (1977), ...   \n",
       "98   [, Star Wars: Episode IV - A New Hope (1977), ...   \n",
       "99   [, Star Wars: Episode IV - A New Hope (1977), ...   \n",
       "100  [, Star Wars: Episode IV - A New Hope (1977), ...   \n",
       "101  [, Star Wars: Episode IV - A New Hope (1977), ...   \n",
       "\n",
       "                                title  flag  not_cold  warm  cold  \n",
       "97        Deconstructing Harry (1997)   1.0         1     1     0  \n",
       "98           Indecent Proposal (1993)   1.0         1     1     0  \n",
       "99                   Big Daddy (1999)   1.0         1     1     0  \n",
       "100      But I'm a Cheerleader (1999)   1.0         1     1     0  \n",
       "101  Desperately Seeking Susan (1985)   1.0         1     1     0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>not_cold</th>\n",
       "      <th>warm</th>\n",
       "      <th>cold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7331.000000</td>\n",
       "      <td>7331.000000</td>\n",
       "      <td>7331.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.566498</td>\n",
       "      <td>0.480426</td>\n",
       "      <td>0.003547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.495592</td>\n",
       "      <td>0.499651</td>\n",
       "      <td>0.059452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          not_cold         warm         cold\n",
       "count  7331.000000  7331.000000  7331.000000\n",
       "mean      0.566498     0.480426     0.003547\n",
       "std       0.495592     0.499651     0.059452\n",
       "min       0.000000     0.000000     0.000000\n",
       "25%       0.000000     0.000000     0.000000\n",
       "50%       1.000000     0.000000     0.000000\n",
       "75%       1.000000     1.000000     0.000000\n",
       "max       1.000000     1.000000     1.000000"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_[['not_cold','warm','cold']].describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_.to_pickle(save_path+\"test_warm_cold_ood2.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.003057"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "7331*0.003547"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/zyang/datasets/ml-1m/test_warm_cold_ood2.pkl'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_path+\"test_warm_cold_ood2.pkl\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "minigpt4",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
